{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "name": "#%%\n"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "from pathlib import Path\n",
    "import pandas as pd\n",
    "\n",
    "root_dir: Path = Path.cwd().parent\n",
    "data_dir: Path = root_dir / 'data'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "train_file = data_dir / 'atis_snips/atis' / 'train.csv'\n",
    "train_intents = Counter(pd.read_csv(train_file)['intent'])\n",
    "\n",
    "dev_file = data_dir / 'atis_snips/atis' / 'dev.csv'\n",
    "dev_intents = Counter(pd.read_csv(dev_file)['intent'])\n",
    "\n",
    "test_file = data_dir / 'atis_snips/atis' / 'test.csv'\n",
    "test_intents = Counter(pd.read_csv(test_file)['intent'])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "name                                        train    dev    test\n",
      "----------------------------------------  -------  -----  ------\n",
      "atis_flight_time                               45      9       1\n",
      "atis_quantity                                  41     10       3\n",
      "atis_ground_service                           230     25      36\n",
      "atis_capacity                                  15      1      21\n",
      "atis_flight#atis_airfare                       19      2      12\n",
      "atis_aircraft                                  70     11       9\n",
      "atis_abbreviation                             130     17      33\n",
      "atis_cheapest                                   1      0       0\n",
      "atis_airline#atis_flight_no                     2      0       0\n",
      "atis_restriction                                5      1       0\n",
      "atis_airfare                                  385     38      48\n",
      "atis_flight                                  3309    357     632\n",
      "atis_aircraft#atis_flight#atis_flight_no        1      0       0\n",
      "atis_ground_service#atis_ground_fare            1      0       0\n",
      "atis_distance                                  17      3      10\n",
      "atis_airport                                   17      3      18\n",
      "atis_city                                      18      1       6\n",
      "atis_ground_fare                               15      3       7\n",
      "atis_meal                                       6      0       6\n",
      "atis_day_name                                   0      0       2\n",
      "atis_airfare#atis_flight                        0      0       1\n",
      "atis_airfare#atis_flight_time                   0      1       0\n",
      "atis_flight_no#atis_airline                     0      0       1\n",
      "atis_airline                                  139     18      38\n",
      "atis_flight_no                                 12      0       8\n",
      "atis_flight#atis_airline                        0      0       1\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "from collections import defaultdict\n",
    "all_intents = list(set(\n",
    "    list(train_intents.keys()) + list(dev_intents.keys()) + list(test_intents.keys())\n",
    "))\n",
    "\n",
    "series = defaultdict(list)\n",
    "for intent in all_intents:\n",
    "    series['name'].append(intent)\n",
    "    series['train'].append(train_intents.get(intent, 0))\n",
    "    series['dev'].append(dev_intents.get(intent, 0))\n",
    "    series['test'].append(test_intents.get(intent, 0))\n",
    "\n",
    "from tabulate import tabulate\n",
    "\n",
    "print(tabulate(series, headers='keys'))"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "334     could you give me the schedule of flights for ...\n",
      "389     please list the flight times from newark to bo...\n",
      "515     please list the flight times from pittsburgh t...\n",
      "667     now i'd like a schedule for the flights on tue...\n",
      "714                   what time does flight aa 459 depart\n",
      "719     i would like the time your earliest flight fro...\n",
      "751     what is delta's schedule of morning flights to...\n",
      "771     please list the flight times from newark to bo...\n",
      "798     what is the schedule of flights from boston to...\n",
      "1092    show me the schedule for airlines leaving pitt...\n",
      "Name: text, dtype: object\n"
     ]
    }
   ],
   "source": [
    "# see the intent: atis_flight_time\n",
    "\n",
    "train_table = pd.read_csv(train_file)\n",
    "\n",
    "sentences = train_table[train_table.intent == 'atis_flight_time']['text']\n",
    "sentences = sentences[:10]\n",
    "print(sentences)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "46\n"
     ]
    }
   ],
   "source": [
    "from typing import List, Dict\n",
    "\n",
    "\n",
    "def get_slot_name(slot: str) -> str:\n",
    "    return slot.replace('B-', \"\").replace('I-', '')\n",
    "\n",
    "\n",
    "\n",
    "def get_sentence_spans(tokens: List[str], slots: List[str]) -> Dict[str, List[str]]:\n",
    "\n",
    "    slot_spans = defaultdict(list)\n",
    "    index = 0\n",
    "    while index < len(tokens):\n",
    "\n",
    "        if slots[index].startswith('B-'):\n",
    "            slot_name = get_slot_name(slots[index])\n",
    "            slot_tokens = [tokens[index]]\n",
    "            index += 1\n",
    "            while index < len(tokens) and slots[index].startswith('I-'):\n",
    "                slot_tokens.append(tokens[index])\n",
    "                index += 1\n",
    "            slot_spans[slot_name] = ' '.join(slot_tokens)\n",
    "        else:\n",
    "            index += 1\n",
    "    return slot_spans\n",
    "\n",
    "\n",
    "series = []\n",
    "max_length = -1\n",
    "for _, row in train_table.iterrows():\n",
    "    tokens, slots = row['text'].split(' '), row['slot'].split(' ')\n",
    "    max_length = max(max_length, len(tokens))\n",
    "\n",
    "    slot_spans = get_sentence_spans(tokens, slots)\n",
    "\n",
    "    for slot_name, spans in slot_spans.items():\n",
    "        series.append({\n",
    "            \"slot_name\": slot_name,\n",
    "            \"tokens\": spans\n",
    "        })\n",
    "\n",
    "\n",
    "pd.DataFrame(series).to_excel(data_dir / 'slot-tokens.xlsx', index=False)\n",
    "\n",
    "print(max_length)"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "name": "#%%\n"
    }
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "name": "python388jvsc74a57bd0dac504a1b45ccde4581675a301266d1beca20c04c7113f5cd8e0d2a32f00ef0c",
   "language": "python",
   "display_name": "Python 3.8.8 64-bit ('torch': conda)"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}